#include<vector>
#include<unordered_map>
#include<map>
#include<string>
using std::vector;
using std::unordered_map;
using std::pair;
using std::string;
class SplitTool;
class Configuration;
class PageLibPreprocessor{
public:
    PageLibPreprocessor(Configuration&);
    void doProcess(string &off_filename,string &page_filename);
    void readInfoFromFile(string);
    void cutRedundantPages(string);
    void buildInvertindexTable();
    //void storeOnDisk();
    uint64_t simHash(string content);
    string rssParse(string);
    string replaceDocId(const string& content, const string& newDocId);

private: 
    unordered_map<int,pair<int,int>> _offsetLib;
    unordered_map<string,vector<pair<int,double>>> _invertIndexTable;
};